from langchain_elasticsearch import ElasticsearchStore
from langchain.embeddings.huggingface import HuggingFaceEmbeddings
embedding = HuggingFaceEmbeddings(model_name=r"E:\models\BAAIbge-large-zh-v1.5",
                                            model_kwargs={'device': "cuda"})
# loaders 
import mimetypes 
import asyncio 
from langchain.document_loaders.base import BaseLoader
from langchain_community.document_loaders import PyPDFium2Loader
documents=PyPDFium2Loader(file_path=r'../demo/华昭府销售说辞初稿-未审定0929.pdf',extract_images=True).load()
# Splitter
from langchain.text_splitter import CharacterTextSplitter
text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=0,is_separator_regex=True)
documents = text_splitter.split_documents(documents)
for k,v in enumerate(documents):
    documents[k].metadata['id']=k

#es
async def loadEsStore():
    elastic_vector_search = ElasticsearchStore(
    es_url="http://192.168.2.46:9200",
    index_name="test_index",
    embedding=embedding
    )
    await elastic_vector_search.aadd_documents(documents=documents)
    # elastic_vector_search.
async def main():
    await loadEsStore()
    
if __name__ == '__main__':
    asyncio.run(main())
    